#!/usr/bin/python
# -*- coding: UTF-8 -*-
from __future__ import division
from __future__ import absolute_import
from __future__ import division, print_function, unicode_literals
###########################
### Autor: Sebastian Enger / M.Sc.
### Copyright: Sebastian Enger
### Licence: Commercial / OneTipp
### Version: 1.0.0c - 2-11-2015@23:53 Uhr
### Contact: sebastian.enger@gmail.com
### OneTipp Text Tool in Python: Main File
###########################
"""
Synonym bzw Wortersetzung parallelisieren für schnellere Verarbeitung und Reaktionszeit des Tools
Antonym Datenbank Entwicklung mit Hilfe gecrawlter Websites
Aufbau einer Datenbank mit einfacher deutscher Sprache
Berechnung des lesbarkeitswerts eines eingabetextes - basierend auf einfachen Texten die "simple German " Datenbank für Austausch nutzen, Wissenschaftliche Texte mit Leipzig und unserer lokalen Synonym Datenbank austauschen
Tests am 29.10.2015:
https://github.com/rsennrich/clevertagger
"""
# https://docs.python.org/2/library/configparser.html
import os
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
sys.path.append('/home/onetipp/python/modules')
os.environ['PYTHON_EGG_CACHE'] = '/home/compress/'
import random
import codecs
import re
import mod
import stopwords
import pprint
import pattern.de
from pattern.de import conjugate
from pattern.de import INFINITIVE, PRESENT, PAST, SG, SUBJUNCTIVE
from textblob_de import TextBlobDE as TextBlob
from textblob_de import PatternTagger
from textblob_de import TextBlobDE
import treetaggerwrapper
from pattern.de import article, DEFINITE, INDEFINITE, FEMALE, OBJECT, gender, MALE, FEMALE, NEUTRAL
# cursorMysql = mod.mysql.cursor()
noDoubleHash = set()
re_match = r"(\?|\.|\!)" # Match: ". WORT"
# # sent_tokenize_list = sent_tokenize(text)
# # Summarize the text first and then work on it
# tSumy = mod.summarizeText(text)
# #tokens = mod.nltk.word_tokenize(tSumy)
# tokens = mod.nltk.sent_tokenize(tSumy, language='german')
# tokensRaw = mod.nltk.word_tokenize(text)
# cursorMysql.execute("SELECT p_articletext FROM (publish_de) ORDER BY RAND() LIMIT 1;")
# cursorMysql.execute("SELECT p_articletext FROM (publish_de) WHERE BINARY `id` = '%s' LIMIT 1;" % (word))
import re
# https://perso.limsi.fr/pointal/doku.php?id=dev:treetaggerwrapper
# https://subversion.renater.fr/ttpw/trunk/treetaggerwrapper.py
# http://treetaggerwrapper.readthedocs.org/en/latest/#polls-of-taggers-process
# result = cursorMysql.fetchall()
# lies die Ein und Ausgabedateien
inputfile = sys.argv[1]
# read file into string
# text = codecs.open(inputfile, "r", encoding='utf-8').read()
text = codecs.open(inputfile, "r").read()
tagger = treetaggerwrapper.TreeTagger(TAGLANG='de', TAGDIR='/home/onetipp/software/treetagger/')
GermanStopwords = stopwords.getGermanStopwords()
GermanSTTLIgnoreTags = stopwords.getSttsIgnoreTags()
tokens = mod.nltk.sent_tokenize(text, language='german')
# http://www.clips.ua.ac.be/pages/pattern-de
list_conjugate = [
"VAFIN",
"VVFIN",
]
list_POS_NENN_VV_LOGIC = [
"VVFIN",
"VVIMP",
"VVINF",
"VVIZU",
"VVPP",
"VAFIN",
"VAIMP",
"VAINF",
"VAPP",
"VMFIN",
"VMINF",
"VMPP",
]
list_POS_NENN_LOGIC = [
"NE",
]
list_POS_ARTIKELCHANGE_LOGIC = [
"der",
"die",
"das",
]
ListFinal = []
appendCurrentWord = 0
noAppendNextFlag = 0
lastWord = ""
currentWord = ""
for i, s in enumerate(tokens):
# print(s,i)
if s is not None:
# print("Satz: ", s)
unicode_text = mod.safe_unicode(s)
# tSumy = mod.summarizeText(r)
tags = tagger.tag_text(unicode_text)
tags2 = treetaggerwrapper.make_tags(tags)
for j, ele in enumerate(tags2):
if ele:
word_tmp = ele[0]
unicode_text = mod.safe_unicode(ele[0])
word = unicode_text.encode('utf-8')
# todo: if POS-TAG==NE and NEXT-POS-TAG == VV VFIN etc then VV or VVFIN bleibt so wie es ist.
# pos_tag = ele[1].encode("ascii")
pos_tag_tmp = ele[1]
unicode_text = mod.safe_unicode(ele[1])
pos_tag = unicode_text.encode('utf-8')
up = j + 1
if (up < len(tags2)):
nextelem = tags2[up]
else:
nextelem is None
if nextelem:
unicode_text2 = mod.safe_unicode(nextelem[0])
wordNext = unicode_text2.encode('utf-8')
unicode_text3 = mod.safe_unicode(nextelem[1])
pos_tagNext = unicode_text3.encode('utf-8')
# print("Current word:", word)
# print("\tCurrent POS:", pos_tag)
# print("Next word:", wordNext)
if pos_tag == "ART" and word.lower() in list_POS_ARTIKELCHANGE_LOGIC:
try:
myGender = gender(wordNext)
currentWord = article(wordNext, gender=myGender)
# print("Austausch:", word, " mit: ", currentWord, "<br>")
word = currentWord
appendCurrentWord = 1
ListFinal.append("<b style=\"color:#C30000;\"><i>")
ListFinal.append(currentWord)
ListFinal.append("</i></b>")
continue
except:
1
# if POS is ART then wordNext -> artice(funktion) und ersetzte ART durch articel()
# wenn aktueller Eintrag ein NE - Eigenname - ist
if pos_tag in list_POS_NENN_LOGIC:
if pos_tagNext in list_POS_NENN_VV_LOGIC:
noAppendNextFlag = 1
lastWord = wordNext
ListFinal.append(word)
# print("Current word:", word)
# print("\tCurrent POS:", pos_tag)
# print("Ignore Next word:", wordNext)
# print("\tNext POS:", pos_tagNext)
continue
if pos_tag not in GermanStopwords and pos_tag not in GermanStopwords:
# print("Pos tag to possible Change:",pos_tag)
# print("Word:", word)
if pos_tag in list_conjugate:
conj_tmp = conjugate(word, PAST, 1, SG, mood=SUBJUNCTIVE)
unicode_text = mod.safe_unicode(conj_tmp)
conj = unicode_text.encode('utf-8')
if noAppendNextFlag == 1:
ListFinal.append(word)
noAppendNextFlag = 0
else:
ListFinal.append("<b style=\"color:#57B055;\"><i>")
ListFinal.append(conj)
ListFinal.append("</i></b>")
continue
# print("Word Past: ", conj, " - Lenght: " ,len(ListFinal) ,"<br />")
else:
1
ListFinal.append(word)
continue
else:
1
ListFinal.append(word)
# https://pypi.python.org/pypi/languagedet
# file schreiben
# readabilityVar = str(mod.textstat.flesch_reading_ease(text))
writeThis = " ".join(ListFinal)
writeThis.encode('utf-8')
with codecs.open("/tmp/onetipp_tmp.txt", 'wb+', encoding='utf-8') as f:
f.write(writeThis)
f.close()
# mod.mysql.commit()
# mod.mysql.close()
#
# mod.sphinx.commit()
# mod.sphinx.close()
exit(0)
"""
The Flesch Reading Ease formula
function name - flesch_reading_ease(text)
returns the Flesch Reading Ease Score. Following table is helpful to access the ease of readability in a document.
90-100 : Very Easy
80-89 : Easy
70-79 : Fairly Easy
60-69 : Standard
50-59 : Fairly Difficult
30-49 : Difficult
0-29 : Very Confusing
"""